{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 加载数据\n",
    "import numpy as np \n",
    "import pandas as pd \n",
    "# dataset = np.loadtxt('data/andmal2020/BenignCSVs/Ben0.csv',dtype=int,skiprows=1,usecols=cols,delimiter=',')\n",
    "\n",
    "# 使用pandas批量读取大文件\n",
    "filelist1 = [\n",
    "    'data/andmal2020/BenignCSVs/Ben0.csv',\n",
    "    'data/andmal2020/BenignCSVs/Ben1.csv',\n",
    "    'data/andmal2020/BenignCSVs/Ben2.csv',\n",
    "    'data/andmal2020/BenignCSVs/Ben3.csv',\n",
    "    'data/andmal2020/BenignCSVs/Ben4.csv'\n",
    "]\n",
    "# 暂时先用10个\n",
    "filelist2 = [\n",
    "    'data/andmal2020/Malicious-CSVs/mal1.csv',\n",
    "    'data/andmal2020/Malicious-CSVs/mal2.csv',\n",
    "    'data/andmal2020/Malicious-CSVs/mal3.csv',\n",
    "    'data/andmal2020/Malicious-CSVs/mal4.csv',\n",
    "    'data/andmal2020/Malicious-CSVs/mal5.csv',\n",
    "    'data/andmal2020/Malicious-CSVs/mal6.csv',\n",
    "    'data/andmal2020/Malicious-CSVs/mal7.csv',\n",
    "    'data/andmal2020/Malicious-CSVs/mal8.csv',\n",
    "    'data/andmal2020/Malicious-CSVs/mal9.csv',\n",
    "    'data/andmal2020/Malicious-CSVs/mal10.csv',\n",
    "    # 'data/andmal2020/Malicious-CSVs/mal11.csv',\n",
    "    # 'data/andmal2020/Malicious-CSVs/mal12.csv',\n",
    "    # 'data/andmal2020/Malicious-CSVs/mal13.csv',\n",
    "    # 'data/andmal2020/Malicious-CSVs/mal14.csv',\n",
    "]\n",
    "chunksize = 5000\n",
    "cols = list(range(2,4))\n",
    "# dataset = pd.read_csv('data/andmal2020/BenignCSVs/Ben0.csv',dtype=int,skiprows=1,iterator=True,chunksize=chunk_size,usecols=cols)\n",
    "\n",
    "\n",
    "def read_chunk(filelist,cols,chunksize):\n",
    "    for file in filelist:\n",
    "        dataset = pd.read_csv(file,dtype=int,skiprows=1,iterator=True,chunksize=chunksize,usecols=cols)\n",
    "        for chunk in dataset:\n",
    "            yield chunk.values[0]\n",
    "\n",
    "ben_iter = read_chunk(filelist1,cols,chunksize)\n",
    "def get_ben(ben_iter):\n",
    "    return next(ben_iter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0.])"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mal_iters = [pd.read_csv(file,dtype=int,skiprows=1,iterator=True,chunksize=chunksize/10,usecols=cols) for file in filelist2]\n",
    "\n",
    "# print(len(mal_iters))\n",
    "\n",
    "def get_mal(mal_iters):\n",
    "    result = np.array([])\n",
    "    for mal_iter in mal_iters:\n",
    "        # print(next(mal_iter).values[0])\n",
    "        result = np.append(result,next(mal_iter).values[0])\n",
    "    return result\n",
    "\n",
    "get_mal(mal_iters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2,)\n",
      "(20,)\n"
     ]
    }
   ],
   "source": [
    "print(get_ben(ben_iter).shape)\n",
    "print(get_mal(mal_iters).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  1,   0, 784,   0,   0,   0,   1,   1,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   1,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = dataset.get_chunk()\n",
    "data.shape\n",
    "data.values[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "benign_file_list=['data/andmal2020/BenignCSVs/Ben0.csv']\n",
    "\n",
    "def read_benign():\n",
    "    pass \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "malware_file_list = ['']\n",
    "def read_malware():\n",
    "    pass\n",
    "dataset = pd.read_csv('data/andmal2020/BenignCSVs/Ben0.csv')\n",
    "# pd.read_csv(,)\n",
    "print(dataset.shape)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.9 64-bit ('pysyft': virtualenvwrapper)",
   "name": "python3810jvsc74a57bd054f0cca124f38538f0af9da5035feb8330e039430efda7f6532c7bec2bbfbd8a"
  },
  "language_info": {
   "name": "python",
   "version": ""
  },
  "orig_nbformat": 2
 },
 "nbformat": 4,
 "nbformat_minor": 2
}